{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "27b0921f-94ba-40a3-a4fb-348ef234a052",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10000 loss 5611.704502\n",
      "epoch 20000 loss 5258.726277\n",
      "epoch 30000 loss 4960.271811\n",
      "epoch 40000 loss 4707.234957\n",
      "epoch 50000 loss 4492.067734\n",
      "epoch 60000 loss 4308.511724\n",
      "epoch 70000 loss 4151.375892\n",
      "epoch 80000 loss 4016.352800\n",
      "epoch 90000 loss 3899.866551\n",
      "five kold cross validation score is 3798.9563843996484\n",
      "valid score is 4214.092765494475\n",
      "epoch 10000 loss 5421.168404\n",
      "epoch 20000 loss 5106.640864\n",
      "epoch 30000 loss 4838.968257\n",
      "epoch 40000 loss 4610.442887\n",
      "epoch 50000 loss 4414.661874\n",
      "epoch 60000 loss 4246.304449\n",
      "epoch 70000 loss 4100.947379\n",
      "epoch 80000 loss 3974.911961\n",
      "epoch 90000 loss 3865.137182\n",
      "five kold cross validation score is 3769.0835464956263\n",
      "valid score is 4615.051403980545\n",
      "epoch 10000 loss 5586.652342\n",
      "epoch 20000 loss 5295.082496\n",
      "epoch 30000 loss 5044.857033\n",
      "epoch 40000 loss 4829.540681\n",
      "epoch 50000 loss 4643.729724\n",
      "epoch 60000 loss 4482.885115\n",
      "epoch 70000 loss 4343.192699\n",
      "epoch 80000 loss 4221.446108\n",
      "epoch 90000 loss 4114.948637\n",
      "five kold cross validation score is 4021.439797419828\n",
      "valid score is 3787.77408008022\n",
      "epoch 10000 loss 5611.711211\n",
      "epoch 20000 loss 5329.043484\n",
      "epoch 30000 loss 5085.368704\n",
      "epoch 40000 loss 4874.725779\n",
      "epoch 50000 loss 4692.097616\n",
      "epoch 60000 loss 4533.259845\n",
      "epoch 70000 loss 4394.653912\n",
      "epoch 80000 loss 4273.280578\n",
      "epoch 90000 loss 4166.610541\n",
      "five kold cross validation score is 4072.518260076695\n",
      "valid score is 3549.9681755234255\n",
      "epoch 10000 loss 5585.410482\n",
      "epoch 20000 loss 5280.637799\n",
      "epoch 30000 loss 5020.541759\n",
      "epoch 40000 loss 4798.003288\n",
      "epoch 50000 loss 4607.068771\n",
      "epoch 60000 loss 4442.757611\n",
      "epoch 70000 loss 4300.901673\n",
      "epoch 80000 loss 4178.011305\n",
      "epoch 90000 loss 4071.163532\n",
      "five kold cross validation score is 3977.917458102055\n",
      "valid score is 3855.0420442947343\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.datasets import load_diabetes\n",
    "\n",
    "class linearegression_model():    \n",
    "    def __init__(self):        \n",
    "        pass\n",
    "\n",
    "    def prepare_data(self):\n",
    "        data = load_diabetes().data\n",
    "        target = load_diabetes().target\n",
    "        X, y = shuffle(data, target, random_state=42)\n",
    "        X = X.astype(np.float32)\n",
    "        y = y.reshape((-1, 1))\n",
    "        data = np.concatenate((X, y), axis=1)        \n",
    "        return data   \n",
    "         \n",
    "    def initialize_params(self, dims):\n",
    "        w = np.zeros((dims, 1))\n",
    "        b = 0\n",
    "        return w, b    \n",
    "    \n",
    "    def linear_loss(self, X, y, w, b):\n",
    "        num_train = X.shape[0]\n",
    "        num_feature = X.shape[1]\n",
    "\n",
    "        y_hat = np.dot(X, w) + b\n",
    "        loss = np.sum((y_hat-y)**2) / num_train\n",
    "        dw = np.dot(X.T, (y_hat - y)) / num_train\n",
    "        db = np.sum((y_hat - y)) / num_train        \n",
    "        return y_hat, loss, dw, db    \n",
    "    \n",
    "    def linear_train(self, X, y, learning_rate, epochs):\n",
    "        w, b = self.initialize_params(X.shape[1])        \n",
    "        for i in range(1, epochs):\n",
    "            y_hat, loss, dw, db = self.linear_loss(X, y, w, b)\n",
    "            w += -learning_rate * dw\n",
    "            b += -learning_rate * db            \n",
    "            if i % 10000 == 0:\n",
    "                print('epoch %d loss %f' % (i, loss))\n",
    "            \n",
    "            params = {                \n",
    "                'w': w,                \n",
    "                'b': b\n",
    "            }\n",
    "            grads = {                \n",
    "                'dw': dw,                \n",
    "                'db': db\n",
    "            }        \n",
    "        return loss, params, grads    \n",
    "    \n",
    "    def predict(self, X, params):\n",
    "        w = params['w']\n",
    "        b = params['b']\n",
    "        y_pred = np.dot(X, w) + b        \n",
    "        return y_pred    \n",
    "        \n",
    "    def linear_cross_validation(self, data, k, randomize=True):        \n",
    "        if randomize:\n",
    "            data = list(data)\n",
    "            shuffle(data)\n",
    "\n",
    "        slices = [data[i::k] for i in range(k)]        \n",
    "        for i in range(k):\n",
    "            validation = slices[i]\n",
    "            train = [data                        \n",
    "            for s in slices if s is not validation for data in s]\n",
    "            train = np.array(train)\n",
    "            validation = np.array(validation)            \n",
    "            yield train, validation\n",
    "            \n",
    "            \n",
    "\n",
    "lr = linearegression_model()\n",
    "data = lr.prepare_data()   \n",
    "\n",
    "for train, validation in lr.linear_cross_validation(data, 5):\n",
    "    X_train = train[:, :10]\n",
    "    y_train = train[:, -1].reshape((-1, 1))\n",
    "    X_valid = validation[:, :10]\n",
    "    y_valid = validation[:, -1].reshape((-1, 1))\n",
    "\n",
    "    loss5 = []\n",
    "    loss, params, grads = lr.linear_train(X_train, y_train, 0.001, 100000)\n",
    "    loss5.append(loss)\n",
    "    score = np.mean(loss5)\n",
    "    print('five kold cross validation score is', score)\n",
    "    y_pred = lr.predict(X_valid, params)\n",
    "    valid_score = np.sum(((y_pred - y_valid) ** 2)) / len(X_valid)\n",
    "    print('valid score is', valid_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a46ec0d-09cc-457f-ab27-7c8d350b90e9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
